clear all

%% PARAMS
opt=0;
% opt=1;
SECTOR_ID=[433]
% SECTOR_ID=[24 29 25]
% % case opt
% SECTOR_ID =[1:100];


%% Chosing stocks in the same sector and equivalent liquidity
all_sec_id_ok = getLiquidSecurityInSector(opt,SECTOR_ID');

%% Get financial ratios from selected stocks
sub_sec_id_ok = all_sec_id_ok(:,1);
[financial_ratio, dataX] = getFIFromSecurities(sub_sec_id_ok);

%% Normalize data X
dataX_normalize = normalizeDataX(dataX);
%%
[mat_X, vec_y, TotalData, dataX_normalize,pricevec] = preparePriceData(dataX_normalize);
[b,~,~,inmodel] = stepwisefit(mat_X,vec_y);
b=b';
YFIT = b(inmodel)*mat_X(:,inmodel)';
% YFIT=maniArray(YFIT,dataX_normalize(:,end));
%%
Testchunk=chunk(dataX_normalize(:,3),dataX_normalize(:,1));
window=0;
mat_x=[];
mat_y=[];
pos1=1;
n = length(Testchunk);
includedLag=1;
for i=1:length(Testchunk)-1
    fprintf('Processing %d / %d\n',i, n)
    pos2=sum(dataX_normalize(1:Testchunk(i+1)-1,end));
    [xvariable yvariable]=PrepareRegressX(TotalData(pos1+...
        (isnan(TotalData(pos1,1))):pos2,:),YFIT(Testchunk(i):Testchunk(i+1)-1),...
        dataX_normalize(Testchunk(i):Testchunk(i+1)-1,end),1);
    pos1=pos2+1;
    if ~isempty(xvariable)&&includedLag~=0        
        xvariable=horzcat(xvariable,ExtractLagVec(yvariable,includedLag));
        
    end
    mat_x=[mat_x;xvariable];
    mat_y=[mat_y;yvariable];
    %     window=[window;[size(xvariable,1) length(yvariable)]];
end
%% Start
[aa bb]=checkoutliner(mat_x,mat_y);
aa=[aa(:,1:10) DataNormalize(aa(:,11:end))];
bb=DataNormalize(bb);
aaa=aa(:,[11:end]);

mat_x = aaa
mat_y = bb
%% Start to use neural network to predict
% %Use neural network to predict 
% inputs = mat_x';
% targets = mat_y';
% hiddenLayerSize = 10;
% net = fitnet(hiddenLayerSize);
% net.divideParam.trainRatio = 70/100;
% net.divideParam.valRatio = 15/100;
% net.divideParam.testRatio = 15/100;
% 
% [net,tr] = train(net,inputs,targets);
% 
% outputs = net(inputs);
% errors = gsubtract(targets,outputs);
% performance = perform(net,targets,outputs);

% stepwise(aa,bb)
% mattest={};
% for i=1:size(aa,2)
%     mattest{i}=find(isnan(aa))
% end